/*
 * Copyright (c) 2016. For Intelligent Group.
 */

package com.intelligent.ispc.core.service.impl;


import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.intelligent.ispc.chart.service.ChartExporter;
import com.intelligent.ispc.chart.service.impl.ChartExporterImpl;
import com.intelligent.ispc.common.mapper.JsonMapper;
import com.intelligent.ispc.common.utils.*;
import com.intelligent.ispc.constant.AppConstant;
import com.intelligent.ispc.constant.ProcessConstant;
import com.intelligent.ispc.core.dto.*;
import com.intelligent.ispc.core.service.FileDataService;
import com.intelligent.ispc.core.service.SpcService;
import com.intelligent.ispc.core.service.SysConfigService;
import com.intelligent.ispc.core.service.SystemService;
import com.intelligent.ispc.export.spc.SpcExportBuilder;
import com.intelligent.ispc.export.spc.SpcExportWorker;
import com.intelligent.ispc.r.dto.*;
import com.intelligent.ispc.r.service.AnalysisService;
import com.intelligent.ispc.utils.ApplicationPathUtil;
import com.intelligent.ispc.utils.ProcessHelper;
import com.intelligent.ispc.utils.PythonUtil;
import com.intelligent.ispc.utils.SpcFilterUtil;
import com.intelligent.ispc.utils.executor.ExportThreadManager;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.File;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 * Created by Julia.Zhou on 2016/08/16.
 */
@Service
public class SpcServiceImpl implements SpcService {

    private final Logger logger = LoggerFactory.getLogger(SpcServiceImpl.class);

    @Autowired
    private FileDataService fileDataService;
    @Autowired
    private SystemService systemService;
    @Autowired
    private AnalysisService analysisService;
    @Autowired
    private SysConfigService sysConfigService;

    private JsonMapper jsonMapper = JsonMapper.nonDefaultMapper();
    private final long digitId = 28L; //digit Number id in table

    @Override
    public FieldDataSourceDto findFieldDataSource(List<String> filePaths) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_DECODE_FILE);
        if (filePaths == null || filePaths.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_11001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_SCRIPT_PATH);
        String scriptPath = null;
        if (sysConfigDto != null && StringUtils.isNotBlank(sysConfigDto.getValue())) {
            scriptPath = sysConfigDto.getValue();
        }
        if (StringUtils.isBlank(scriptPath)) {
            scriptPath = ApplicationPathUtil.getPath(AppConstant.PATH_SCRIPT, AppConstant.PATH_SCRIPT_FILENAME);
        }

        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));

        processHelper.postProcess(28, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION));
        PythonUtil util = new PythonUtil("spc");
        List<String> searchKeys = util.getSearchKeys(filePaths, scriptPath);
        processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION_DONE));

        Set<String> searchKeyNames = new LinkedHashSet<>();
        FieldDataSourceDto fieldDataSourceDto = new FieldDataSourceDto();
        fieldDataSourceDto.setSearchFields(this.getSearchKeys(searchKeys, searchKeyNames, systemConfigDto));

        List<String> attributeKeys = getAttributeKeys(util.getAttributeKeys(filePaths, scriptPath), systemConfigDto);

        // searchKeys and attributeKeys
        //check if keys which come from files configuration all exist in files.
        //after check,(if configuration is right )the param keys will keep null,otherwise it will keep the false keys names
        //
//        Map<String, String> resultMap = new HashMap<>();
        String checkResult = "";
        for (String path : filePaths) {
            List<String> singleFile = new ArrayList<>();
            singleFile.add(path);
            List<String> allColNames = getAllColNames(jsonMapper.fromJson(util.getColumnKeys(singleFile, false).toString(), List.class));
            String attributeCheckResult = checkKeys(attributeKeys, allColNames);
            String searchCheckResult = checkKeys(new ArrayList<>(searchKeyNames), allColNames);
            String result = "";
            if (!searchCheckResult.isEmpty()) {
                result += searchCheckResult;
            }
            if (!attributeCheckResult.isEmpty()) {
                result += attributeCheckResult;
            }
            if (!result.isEmpty()) {
                String fileName = path.substring(path.lastIndexOf(File.separator) + 1, path.length());
                checkResult += fileName + "\n" + result;
            }
            if (!result.isEmpty()) {
                checkResult += "\n";
            }
        }
        if (!checkResult.isEmpty()) {
            String[] args = new String[]{checkResult};
            throw new ApplicationException(ExceptionMessages.ERR_11002, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_COLUMN_INVALID, args));
//            throw new ApplicationException("These columns from configuration not exist in file : " + searchCheckResult + attributeCheckResult);
        }

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        List<String> itemNames = util.getColumnKeys(filePaths, scriptPath);
        fieldDataSourceDto.setTestItemNames(this.getTestItemKeys(itemNames));
        processHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));

        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));

        return fieldDataSourceDto;
    }


    @Override
    public List<SpcResultDto> getStatisticResult(List<KeyValueDto> filePath, Map<String, List<DynSearchFieldDto>> searchKeys, List<String> testItemNames,
                                                 Map<String, List<TestItemDto>> excludeItems, Map<String, NDStandardDto> param, Map<String, List<String>> chartNames) {
        if (filePath == null || filePath.isEmpty() || testItemNames == null || testItemNames.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_11001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_PARAMETER_INVALID));
        }

        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXECUTE_TASK);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION));
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        String primaryKey = systemConfigDto.getPrimaryKey();
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION_DONE));

        List<String> targetFilePath = getFilePath(filePath);

        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        List<String> columnNames = Lists.newLinkedList();
        int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());

        SpcResultDto spcResultDto;
        columnNames.addAll(testItemNames);
        for (String testItemName : testItemNames) {
            if (searchKeys == null) {
                break;
            }
            List<DynSearchFieldDto> searchKeysList = searchKeys.get(testItemName);
            if (searchKeysList != null) {
                columnNames.addAll(new SpcFilterUtil().parseSearchKey(searchKeys.get(testItemName)));
            }
        }
        processHelper.postProcess(30, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        Map<String, Object> jsonData = parseValueData(targetFilePath, columnNames);
        processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));
        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        for (String testItemName : testItemNames) {
            List<DynSearchFieldDto> searchKeysList = getValueFromMap(searchKeys, testItemName);
            List<TestItemDto> excludeItemsList = getValueFromMap(excludeItems, testItemName);
            NDStandardDto curParam = getValueFromMap(param, testItemName);
            List<String> chartNamesList = getValueFromMap(chartNames, testItemName);
            spcResultDto = getSpcResult(jsonData, targetFilePath, testItemName, primaryKey, excludeItemsList, curParam, true, searchKeysList, digitNum, chartNamesList);
            if (spcResultDto == null) {
                logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName + ".It's null");
                continue;
            }
            spcResultDto.setPrimaryKey(primaryKey);
            spcResultDto.setTestItemName(testItemName);
            spcResultDtos.add(spcResultDto);
            logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName);
        }
        processHelper.postProcess(95, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));
        return spcResultDtos;
    }

    @Override
    public List<SpcResultDto> getStatisticResult(List<KeyValueDto> filePath, Map<String, SpcExportDto> exportDtos, List<String> testItemNames) {
        if (filePath == null || filePath.isEmpty() || testItemNames == null || testItemNames.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_11001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_PARAMETER_INVALID));
        }

        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXECUTE_TASK);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION));
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        String primaryKey = systemConfigDto.getPrimaryKey();
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION_DONE));

        List<String> targetFilePath = getFilePath(filePath);

        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        List<String> columnNames = Lists.newLinkedList();
        int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());

        SpcResultDto spcResultDto;
        columnNames.addAll(testItemNames);
        for (String testItemName : testItemNames) {
            SpcExportDto spcExportDto = exportDtos.get(testItemName);
            List<DynSearchFieldDto> searchKeysList = spcExportDto.getSearchKeys();
            if (searchKeysList != null) {
                columnNames.addAll(new SpcFilterUtil().parseSearchKey(searchKeysList));
            }
        }
        processHelper.postProcess(30, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        Map<String, Object> jsonData = parseValueData(targetFilePath, columnNames);
        processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));
        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        for (String testItemName : testItemNames) {
            SpcExportDto spcExportDto = exportDtos.get(testItemName);
            List<DynSearchFieldDto> searchKeysList = spcExportDto.getSearchKeys();
            List<TestItemDto> excludeItemsList = spcExportDto.getExcludeItems();
            NDStandardDto curParam = spcExportDto.getParam();
            List<String> chartNamesList = spcExportDto.getChartNames();
            spcResultDto = getSpcResult(jsonData, targetFilePath, testItemName, primaryKey, excludeItemsList, curParam, true, searchKeysList, digitNum, chartNamesList);
            if (spcResultDto == null) {
                logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName + ".It's null");
                continue;
            }
            spcResultDto.setPrimaryKey(primaryKey);
            spcResultDto.setTestItemName(testItemName);
            spcResultDtos.add(spcResultDto);
            logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName);
        }
        processHelper.postProcess(95, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));
        return spcResultDtos;
    }

    @Override
    public List<SpcResultDto> getStatisticResult(List<KeyValueDto> filePath, List<DynSearchFieldDto> searchKeys, List<String> testItemNames, List<TestItemDto> excludeItems, NDStandardDto param, List<String> chartNames) {
        if (filePath == null || filePath.isEmpty() || testItemNames == null || testItemNames.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_11001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_PARAMETER_INVALID));
        }

        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXECUTE_TASK);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION));
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        String primaryKey = systemConfigDto.getPrimaryKey();
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_SPC_INITIAL_CONFIGUTATION_DONE));

        List<String> targetFilePath = getFilePath(filePath);

        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        int testItemNum = testItemNames.size();
        List<String> columnNames = Lists.newLinkedList();
        columnNames.addAll(testItemNames);
        columnNames.addAll(new SpcFilterUtil().parseSearchKey(searchKeys));
        int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());

        if (testItemNum == 1) {
            processHelper.postProcess(30, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
            String testItemName = testItemNames.get(0);
            Map<String, Object> jsonData = parseValueData(targetFilePath, columnNames);
            processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));
            processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
            List<String> curItemChartNames = chartNames;
            if (chartNames == null) {
                curItemChartNames = new ArrayList<>();
                curItemChartNames.add(AppConstant.CHART_TYPE_SPC_SCT);
                curItemChartNames.add(AppConstant.CHART_TYPE_SPC_SD);
                curItemChartNames.add(AppConstant.CHART_TYPE_SPC_R);
                curItemChartNames.add(AppConstant.CHART_TYPE_SPC_XBAR);

            }
            SpcResultDto spcResultDto = getSpcResult(jsonData, targetFilePath, testItemName, primaryKey, excludeItems, param, true, searchKeys, digitNum, curItemChartNames);
            if (spcResultDto == null) {
                logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName + ".It's null");
                return null;
            }
            spcResultDto.setPrimaryKey(primaryKey);
            spcResultDto.setTestItemName(testItemName);
            spcResultDtos.add(spcResultDto);
            processHelper.postProcess(95, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
            logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName);
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));

        } else {
            SpcResultDto spcResultDto;
            Map<String, Object> jsonData = parseValueData(targetFilePath, columnNames);
            for (String testItemName : testItemNames) {
                spcResultDto = getSpcResult(jsonData, targetFilePath, testItemName, primaryKey, null, null, false, searchKeys, digitNum, chartNames);
                if (spcResultDto == null) {
                    logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName + ".It's null");
                    continue;
                }
                spcResultDto.setPrimaryKey(primaryKey);
                spcResultDto.setTestItemName(testItemName);
                spcResultDtos.add(spcResultDto);
                logger.debug("Got statistics result : spcResultDto with testItem:" + testItemName);
            }
        }

        return spcResultDtos;
    }

    @Override
    public String exportMultipleNDChart(List<KeyValueDto> files, List<DynSearchFieldDto> searchKeys, List<String> testItemName) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXPORT_FILE);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));
        if (files == null || files.size() == 0 || testItemName == null || testItemName.size() == 0) {
            throw new ApplicationException(ExceptionMessages.ERR_11004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_EXPORT_PARAMETER_INVALID));
        }

        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        String[] basePath = new String[1];
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH);
        if (sysConfigDto != null) {
            basePath[0] = sysConfigDto.getValue();
            if (StringUtils.isNotBlank(basePath[0]) && !basePath[0].endsWith("/")) {
                basePath[0] += "/";
            }
        }
        if (StringUtils.isBlank(basePath[0])) {
            basePath[0] = ApplicationPathUtil.getCanonicalPath() + "export/";
        }
        sysConfigDto = null;
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CACHE));

        processHelper.postProcess(21, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
        processHelper.postProcess(basePath[0] + "spc/");

        String dirName = "multipleItem_" + getTimeString();
        String dirSavePath = basePath[0] + "spc/" + dirName;
        String imageDir = basePath[0] + "spc/temp";
        FileUtils.createDir(dirSavePath);
        processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + dirName);

        int rate = 1, readPieceSize = systemService.findSystemConfig().getSpcExportNumber();
        boolean ifSucceed = false;
        ExportManager exportManager = new ExportManager(dirName, dirSavePath, imageDir, rate, readPieceSize);
        ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
        Callable<Boolean> callableTask = () -> exportMultipleNdcChart(0, files, searchKeys, testItemName, exportManager, processHelper, exportThreadManager);
        FutureTask<Boolean> futureTask = exportThreadManager.initSpcExportThread("task", callableTask);
        exportThreadManager.startSpcExportThread();
        try {
            ifSucceed = futureTask.get();
        } catch (InterruptedException e) {
            futureTask.cancel(true);
        } catch (ExecutionException e) {
            e.printStackTrace();
        }

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
        FileUtils.deleteDir(imageDir);
        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
        if (ifSucceed) {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + dirSavePath);
        } else {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
        }


        return dirSavePath;
    }

    @Override
    public String exportMultipleItem(List<KeyValueDto> files, Map<String, List<DynSearchFieldDto>> searchKeys, List<String> testItemName, Map<String, List<String>> chartNames, Map<String, Map<String, ChartDto>> guidesParams,
                                     Map<String, List<TestItemDto>> excludeItems, Map<String, NDStandardDto> param) {
        if (testItemName.size() != chartNames.size()) {
            throw new ApplicationException(ExceptionMessages.ERR_11001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_PARAMETER_INVALID));
        }

        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXPORT_FILE);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));
        if (files == null || files.size() == 0 || testItemName == null || testItemName.size() == 0) {
            throw new ApplicationException(ExceptionMessages.ERR_11004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_EXPORT_PARAMETER_INVALID));
        }

        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        String[] basePath = new String[1];
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH);
        if (sysConfigDto != null) {
            basePath[0] = sysConfigDto.getValue();
            if (StringUtils.isNotBlank(basePath[0]) && !basePath[0].endsWith("/")) {
                basePath[0] += "/";
            }
        }
        if (StringUtils.isBlank(basePath[0])) {
            basePath[0] = ApplicationPathUtil.getCanonicalPath() + "export/";
        }
        sysConfigDto = null;
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CACHE));

        processHelper.postProcess(21, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
        processHelper.postProcess(basePath[0] + "spc/");

        String dirName = "multipleItem_" + getTimeString();
        String dirSavePath = basePath[0] + "spc/" + dirName;
        String imageDir = basePath[0] + "spc/temp";
        FileUtils.createDir(dirSavePath);
        processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + dirName);

        int rate = 1, readPieceSize = systemService.findSystemConfig().getSpcExportNumber();
        boolean ifSucceed = false;
        ExportManager exportManager = new ExportManager(dirName, dirSavePath, imageDir, rate, readPieceSize, chartNames, guidesParams, excludeItems, param);
        ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
        Callable<Boolean> callableTask = () -> exportMultipleNdcChart(0, files, searchKeys, testItemName, exportManager, processHelper, exportThreadManager);
        FutureTask<Boolean> futureTask = exportThreadManager.initSpcExportThread("task", callableTask);
        exportThreadManager.startSpcExportThread();
        try {
            ifSucceed = futureTask.get();
        } catch (InterruptedException e) {
            futureTask.cancel(true);
        } catch (ExecutionException e) {
            e.printStackTrace();
        }

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
        FileUtils.deleteDir(imageDir);
        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
        if (ifSucceed) {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + dirSavePath);
        } else {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
        }


        return dirSavePath;
    }

    @Override
    public String exportMultipleItem(List<KeyValueDto> files, Map<String, SpcExportDto> exportDtos) {

        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXPORT_FILE);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));
        if (files == null || files.size() == 0) {
            throw new ApplicationException(ExceptionMessages.ERR_11004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_EXPORT_PARAMETER_INVALID));
        }

        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        String[] basePath = new String[1];
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH);
        if (sysConfigDto != null) {
            basePath[0] = sysConfigDto.getValue();
            if (StringUtils.isNotBlank(basePath[0]) && !basePath[0].endsWith("/")) {
                basePath[0] += "/";
            }
        }
        if (StringUtils.isBlank(basePath[0])) {
            basePath[0] = ApplicationPathUtil.getCanonicalPath() + "export/";
        }
        sysConfigDto = null;
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CACHE));

        processHelper.postProcess(21, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
        processHelper.postProcess(basePath[0] + "spc/");

        String dirName = "multipleItem_" + getTimeString();
        String dirSavePath = basePath[0] + "spc/" + dirName;
        String imageDir = basePath[0] + "spc/temp";
        FileUtils.createDir(dirSavePath);
        processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + dirName);

        List<String> testItemName = new ArrayList<>(exportDtos.keySet());
        int rate = 1, readPieceSize = systemService.findSystemConfig().getSpcExportNumber();
        boolean ifSucceed = false;
        ExportManager exportManager = new ExportManager(dirName, dirSavePath, imageDir, rate, readPieceSize);
        ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
        Callable<Boolean> callableTask = () -> exportMultipleNdcCharts(0, files, exportDtos, testItemName, exportManager, processHelper, exportThreadManager);
        FutureTask<Boolean> futureTask = exportThreadManager.initSpcExportThread("task", callableTask);
        exportThreadManager.startSpcExportThread();
        try {
            ifSucceed = futureTask.get();
        } catch (InterruptedException e) {
            futureTask.cancel(true);
        } catch (ExecutionException e) {
            e.printStackTrace();
        }

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
        FileUtils.deleteDir(imageDir);
        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
        if (ifSucceed) {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + dirSavePath);
        } else {
            processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
        }


        return dirSavePath;
    }

    @Override
    public String exportAllChart(List<KeyValueDto> files, List<DynSearchFieldDto> searchKeys, String testItemName, List<String> chartNames,
                                 Map<String, ChartDto> guidesParams, List<TestItemDto> excludeItems, NDStandardDto param) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.SPC_EXPORT_FILE);
        processHelper.postProcess(0, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));

        if (files == null || files.size() == 0 || testItemName == null || testItemName.length() == 0 || chartNames == null || chartNames.size() == 0) {
            throw new ApplicationException(ExceptionMessages.ERR_11004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_SPC_EXPORT_PARAMETER_INVALID));
        }
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        List<String> testItemNameList = new ArrayList<>();
        testItemNameList.add(testItemName);
        SpcResultDto spcResultDto = getStatisticResult(files, searchKeys, testItemNameList, excludeItems, param, chartNames).get(0);
        if (spcResultDto == null) {
            return null;
        }
        List<TestItemDto> testItemDtos = spcResultDto.getTestItems();
        Map<String, Object> chartDatas = spcResultDto.getChartData();
        String basePath = null;
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH);
        if (sysConfigDto != null) {
            basePath = sysConfigDto.getValue();
            if (StringUtils.isNotBlank(basePath) && !basePath.endsWith("/")) {
                basePath += "/";
            }
        }
        processHelper.postProcess(30, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CACHE));
        processHelper.postProcess(31, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
        if (StringUtils.isBlank(basePath)) {
            basePath = ApplicationPathUtil.getCanonicalPath() + "export/";
        }
        processHelper.postProcess(basePath + "spc/");

        String fileName = "singleItem_" + getTimeString() + ".xlsx";
        String savePath = basePath + "spc/" + fileName;
        FileUtils.createParentPath(savePath);
        String imageDir = basePath + "spc/temp";
        FileUtils.createDir(imageDir);
        processHelper.postProcess(40, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + fileName);

        Map<String, Object> searchParams = new HashMap<>();
        Map<String, Object> results = new HashMap<>();
        Map<String, Object> charts = new HashMap<>();
        searchParams.put("exportCharts", charts);
        searchParams.put("itemName", testItemName);
        searchParams.put("primaryKey", spcResultDto.getPrimaryKey());

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA));
        Map<String, Object> tempSearchParams;
        Map<String, Object> tempResults;
        tempResults = getRuleDescription();
        results.putAll(tempResults);
        //get the sheet page with all the types of charts
        Map<String, Map> tempReturnValue;
        tempReturnValue = getChartPage(chartDatas, testItemName, chartNames, imageDir, guidesParams);
        searchParams.putAll(tempReturnValue.get("searchParams"));
        results.putAll(tempReturnValue.get("results"));
        //get the sheet page with the data table
        tempResults = getTablePage(chartDatas, testItemDtos);
        results.putAll(tempResults);
        processHelper.postProcess(80, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));

        SpcExportWorker spcExportWorker = new SpcExportWorker();
        spcExportWorker.initWorkbook(savePath);
        spcExportWorker.setDigNum(Integer.parseInt(systemService.findSpcConfigById(digitId).getValue()));
        spcExportWorker.buildSPC4SingleItem(searchParams, results);
        SpcExportBuilder spcExportBuilder = new SpcExportBuilder();
        spcExportBuilder.drawSpcExcel(savePath, spcExportWorker);
        processHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
        FileUtils.deleteDir(imageDir);

        logger.debug("Exported xls file at: " + savePath);
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + savePath);
        return savePath;
    }

    /*get test items' json data by item names and file names*/
    private Map<String, Object> parseValueData(List<String> filePath, List<String> itemNames) {
        Map<String, List> inputParams = Maps.newHashMap();
        inputParams.put("filePaths", filePath);
        inputParams.put("itemNames", itemNames);
        try {
            SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_SCRIPT_PATH);
            String scriptPath = null;
            if (sysConfigDto != null && StringUtils.isNotBlank(sysConfigDto.getValue())) {
                scriptPath = sysConfigDto.getValue();
            } else {
                scriptPath = ApplicationPathUtil.getPath(AppConstant.PATH_SCRIPT, AppConstant.PATH_SCRIPT_FILENAME);
            }
            PythonUtil util = new PythonUtil("spc");
            Map<String, List<String>> valueData = util.getDataByName(filePath, itemNames, scriptPath);
            List<String> paramData = util.getParamKeys(filePath);

            Map<String, Object> valueAndParam = new HashMap<>();
            valueAndParam.put("valueData", valueData);
            valueAndParam.put("paramData", paramData);

            return valueAndParam;
        } catch (Exception e) {
            e.printStackTrace();
            throw new ApplicationException(ExceptionMessages.ERR_15003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GLOBAL_PYTHON_SCRIPT_ERROR));
        }
    }

    /*translate filepath type from KeyValueDto to String*/
    private List<String> getFilePath(List<KeyValueDto> filePath) {
        List<String> targetFilePaths = new ArrayList<>();
        for (KeyValueDto keyValueDto : filePath) {
            targetFilePaths.add(keyValueDto.getValue().toString());
        }
        return targetFilePaths;

    }

    /*get param:usl,lsl and unit...*/
    private Map<String, Object> getTestItemParam(Map<String, Object> paramData, String testItemName, NDStandardDto param) {
        Map<String, Object> testItemParam = new HashMap<>();
        Object paramValueObject = paramData.get(testItemName);
        String paramValue = null;
        if (paramValueObject != null) {
            paramValue = paramValueObject.toString();
        }
        if (paramValue != null) {
            String[] splitedValues = paramValue.split(",", -1);
            String usl = null;
            String lsl = null;
            String unitName = null;
            if (param == null) {
                if (splitedValues != null && splitedValues.length == 3) {
                    String uslStr = splitedValues[0];
                    String lslStr = splitedValues[1];
                    unitName = splitedValues[2];
                    if (StringUtils.isNumeric(uslStr)) {
                        usl = uslStr;
                    }
                    if (StringUtils.isNumeric(lslStr)) {
                        lsl = lslStr;
                    }
                }
            } else {
                Object uslObject = param.getUslValue();
                Object lslObject = param.getLslValue();
                if (uslObject != null) {
                    usl = uslObject.toString();
                }
                if (lslObject != null) {
                    lsl = lslObject.toString();
                }
            }
            testItemParam.put("Name", testItemName);
            testItemParam.put(AppConstant.GLOBAL_USL, usl);
            testItemParam.put(AppConstant.GLOBAL_LSL, lsl);
            testItemParam.put(AppConstant.GLOBAL_UNIT, unitName);
            return testItemParam;
        } else {
            return null;
        }

    }

    /*parse test item json data*/
    private SpcResultDto getSpcResult(Map<String, Object> jsonData, List<String> targetFilePath, String testItemName, String primaryKey, List<TestItemDto> excludeItems,
                                      NDStandardDto param, boolean multiCharts, List<DynSearchFieldDto> searchKeys, Integer digitNum, List<String> chartNames) {
        SpcResultDto spcResultDto = new SpcResultDto();
        int fileCounter = targetFilePath.size();

        List<Double> paintData = new ArrayList<>();
        //take the param of the first file for all the files
        Map<String, Object> testItemParam = new HashMap<>();
        List<String> paramDataList = (List<String>) jsonData.get("paramData");
        for (int i = 0; i < fileCounter; i++) {
            Map<String, Object> params = jsonMapper.fromJson(paramDataList.get(i), Map.class);
            testItemParam = getTestItemParam(params, testItemName, param);
            if (testItemParam != null) {
                break;
            }
            params = null;
        }
        if (testItemParam == null) {
            testItemParam = new HashMap<>();
        }
        List<TestItemDto> testItemDtos = new ArrayList<>(4000);
        //if excludeItems exist, get all the items' id
        Set<Long> excludeIds;
        boolean ifExclude = (excludeItems != null && !excludeItems.isEmpty());
        excludeIds = new LinkedHashSet<>();
        if (ifExclude) {
            excludeIds.addAll(excludeItems.stream().map(TestItemDto::getId).collect(Collectors.toList()));
        }
        //parse file one by one
        int testItemCounter = 0;
        String valueJsonName = "valueData";
        jsonData.get(valueJsonName);
        Map<String, List> valueMap = (Map<String, List>) jsonData.get(valueJsonName);
        for (int i = 0; i < fileCounter; i++) {
            String infoJsonName = "DATA_INFO" + i;
            Stream<String> parsedTestItemsStr;
            try {
                parsedTestItemsStr = valueMap.get(infoJsonName).stream();
            } catch (NullPointerException e) {
                continue;
            }
            List<Map> parsedTestItems = new ArrayList<>(4000);
            parsedTestItemsStr.forEach((itemStr) -> parsedTestItems.add(jsonMapper.fromJson(itemStr, Map.class)));
            parsedTestItemsStr.close();
            // set aim testItemDtoList and get paintData Array for painting
            for (Map rowData : parsedTestItems) {
                //Filter data
                if (searchKeys != null && !searchKeys.isEmpty()) {
                    SpcFilterUtil spcFilterUtil = new SpcFilterUtil();
                    if (!spcFilterUtil.filterData(searchKeys, rowData)) {
                        continue;
                    }
                }
                Object valueObject = rowData.get(testItemName);
                //judge if testItem exists in current file
                if (valueObject == null) {
                    break;
                }
                //if the testItem value is not numeric,pass and record the next testItem value
                String value = StringUtils.formatBigDecimal(valueObject.toString());

                boolean isNumeric = StringUtils.isNumeric(value);
                TestItemDto testItemDto = new TestItemDto();
                Long id = Long.valueOf(testItemCounter++);
                testItemDto.setId(id);
                testItemDto.setName(testItemName);
                Object primaryKeyValue = rowData.get(primaryKey);
                String localPriamryValue = null;
                if (primaryKeyValue != null) {
                    localPriamryValue = primaryKeyValue.toString();
                }
                if (primaryKeyValue == null || localPriamryValue.isEmpty()) {
                    continue;
                }

                testItemDto.setPrimaryValue(localPriamryValue);
                testItemDto.setValue(value);
                Object lslObject = testItemParam.get(AppConstant.GLOBAL_LSL);
                Object uslObject = testItemParam.get(AppConstant.GLOBAL_USL);
                if (lslObject != null) {
                    testItemDto.setLsl(lslObject.toString());
                } else {
                    testItemDto.setLsl(null);
                }
                if (uslObject != null) {
                    testItemDto.setUsl(uslObject.toString());
                } else {
                    testItemDto.setUsl(null);
                }
                Object unitNameObject = testItemParam.get(AppConstant.GLOBAL_UNIT);
                if (unitNameObject != null) {
                    testItemDto.setUnitName(unitNameObject.toString());
                } else {
                    testItemDto.setUnitName(null);
                }
                testItemDto.setNumeric(isNumeric);
                testItemDto.setPass(null);
                if (isNumeric) {
                    double doubleValue = Double.parseDouble(value);
                    //if excludeItems is not null , pass the specific testItem
                    if (!ifExclude || !excludeIds.contains(id)) {
                        paintData.add(doubleValue);
                    }
                    String usl = testItemDto.getUsl(), lsl = testItemDto.getLsl();
                    if (lsl != null && usl != null) {
                        double uslValue = Double.parseDouble(usl),
                                lslValue = Double.parseDouble(lsl);
                        if (doubleValue >= lslValue && doubleValue <= uslValue) {
                            testItemDto.setPass(true);
                        } else {
                            testItemDto.setPass(false);
                        }
                    } else if (usl != null) {
                        double uslValue = Double.parseDouble(usl);
                        if (doubleValue > uslValue) {
                            testItemDto.setPass(false);
                        } else {
                            testItemDto.setPass(true);
                        }
                    } else if (lsl != null) {
                        double lslValue = Double.parseDouble(lsl);
                        if (doubleValue < lslValue) {
                            testItemDto.setPass(false);
                        } else {
                            testItemDto.setPass(true);
                        }
                    }

                }
                testItemDtos.add(testItemDto);
            }
        }
        if (paintData.isEmpty()) {
            return null;
        }
        //set charData and init warning rules
        List<CPWarningRuleDto> cpWarningRuleDtos = new ArrayList<>();
        List<SpcConfigDto> spcConfigDtos = new ArrayList<>();
        getAllWarningRules(cpWarningRuleDtos, spcConfigDtos);
        AnalysisParamDto analysisParamDto = getAnalysisParamDto(paintData, testItemParam);
        if (digitNum != null) {
            analysisParamDto.setDigitNum(digitNum);
        }
        Map<String, Object> chartData = new HashMap<>();
        int defaultGroupValue = analysisParamDto.getGroupNum();


        if (analysisParamDto.getDataCount() != 0) {
            //get NormalDistributionDto
            Integer groupNumber = updateGroupNumber("ndc");
            if (groupNumber != null) {
                analysisParamDto.setGroupNum(groupNumber);
            }
            if (param != null && param.getGroupValue() != null) {
                analysisParamDto.setGroupNum(param.getGroupValue());
            }
            NormalDistributionDto normalDistributionDto = analysisService.analyseNormalDistribution(analysisParamDto, cpWarningRuleDtos);
            chartData.put(AppConstant.CHART_TYPE_SPC_ND, normalDistributionDto);

            //get ControlChartDto with 5 kind of different types
            if (multiCharts && chartNames != null) {
                if (chartNames.contains(AppConstant.CHART_TYPE_SPC_SCT)) {
                    ScatterDto scatterDto = analysisService.analyseScatter(analysisParamDto, cpWarningRuleDtos);
                    chartData.put(AppConstant.CHART_TYPE_SPC_SCT, scatterDto);
                }
                analysisParamDto.setGroupNum(defaultGroupValue);
                groupNumber = updateGroupNumber("cc");
                if (groupNumber != null) {
                    analysisParamDto.setGroupNum(groupNumber);
                }
                if (chartNames.contains(AppConstant.CHART_TYPE_SPC_XBAR)) {
                    ControlDataDto controlDataDtoCM = analysisService.analyseControlChart(AppConstant.CHART_TYPE_SPC_XBAR, analysisParamDto, spcConfigDtos);
                    chartData.put(AppConstant.CHART_TYPE_SPC_XBAR, controlDataDtoCM);
                }
                if (chartNames.contains(AppConstant.CHART_TYPE_SPC_R)) {
                    ControlDataDto controlDataDtoCRG = analysisService.analyseControlChart(AppConstant.CHART_TYPE_SPC_R, analysisParamDto, spcConfigDtos);
                    chartData.put(AppConstant.CHART_TYPE_SPC_R, controlDataDtoCRG);
                }
                if (chartNames.contains(AppConstant.CHART_TYPE_SPC_SD)) {
                    ControlDataDto controlDataDtoCS = analysisService.analyseControlChart(AppConstant.CHART_TYPE_SPC_SD, analysisParamDto, spcConfigDtos);
                    chartData.put(AppConstant.CHART_TYPE_SPC_SD, controlDataDtoCS);
                }
                ControlDataDto controlDataDtoCMR = analysisService.analyseControlChart(AppConstant.CHART_TYPE_SPC_MR, analysisParamDto, spcConfigDtos);
                chartData.put(AppConstant.CHART_TYPE_SPC_MR, controlDataDtoCMR);

                ControlDataDto controlDataDtoCR = analysisService.analyseControlChart(AppConstant.CHART_TYPE_SPC_RUN, analysisParamDto, spcConfigDtos);
                chartData.put(AppConstant.CHART_TYPE_SPC_RUN, controlDataDtoCR);


            }
        }

        spcResultDto.setChartData(chartData);
        spcResultDto.setTestItems(testItemDtos);

        return spcResultDto;
    }

    /*package test item data into analysis dto*/
    private AnalysisParamDto getAnalysisParamDto(List<Double> data, Map<String, Object> testItemParam) {
        AnalysisParamDto analysisParamDto = new AnalysisParamDto();
        int length = data.size();
        double[] dataArray = new double[length];
        for (int i = 0; i < length; i++) {
            dataArray[i] = data.get(i);
        }
        analysisParamDto.setAnalysisData(dataArray);
        Object lslObject = testItemParam.get(AppConstant.GLOBAL_LSL);
        Object uslObject = testItemParam.get(AppConstant.GLOBAL_USL);
        if (lslObject != null) {
            String lslStr = lslObject.toString();
            double lsl = Double.parseDouble(lslStr);
            analysisParamDto.setLslValue(lsl);
        } else {
            analysisParamDto.setLslValue(null);
        }
        if (uslObject != null) {
            String uslStr = uslObject.toString();
            double usl = Double.parseDouble(uslStr);
            analysisParamDto.setUslValue(usl);
        } else {
            analysisParamDto.setUslValue(null);
        }

        int groupNum = (int) Math.ceil(Math.sqrt(length));

        analysisParamDto.setGroupNum(groupNum);
        return analysisParamDto;
    }

    /*get warning rules from database and package them into dto*/
    private void getAllWarningRules(List<CPWarningRuleDto> cpWarningRuleDtos, List<SpcConfigDto> spcConfigDtos) {
        List<SpcConfigDto> spcConfigs = systemService.findSpcConfigs();
        String spcConfigType;
        CPWarningRuleDto cpWarningRuleDto;
        SpcConfigDto spcConfigDto;

        for (SpcConfigDto spcConfig : spcConfigs) {
            spcConfigType = spcConfig.getConfigType();
            if (spcConfigType.equals(AppConstant.SPC_CONFIG_CPW) && cpWarningRuleDtos != null) {
                String[] values = spcConfig.getValue().split(",");
                cpWarningRuleDto = new CPWarningRuleDto();
                cpWarningRuleDto.setIndiceName(spcConfig.getName());
                cpWarningRuleDto.setExcellentBoundaryValue(values[0]);
                cpWarningRuleDto.setGoodBoundaryValue(values[1]);
                cpWarningRuleDto.setAcceptableBoundaryValue(values[2]);
                cpWarningRuleDto.setRectificationBoundaryValue(values[3]);
                cpWarningRuleDtos.add(cpWarningRuleDto);

            } else if (spcConfigType.equals(AppConstant.SPC_CONFIG_CCW) && spcConfigDtos != null) {
                spcConfigDto = new SpcConfigDto();
                BeanUtils.copyProperties(spcConfig, spcConfigDto);
                spcConfigDtos.add(spcConfigDto);
            }
        }
    }

    /*package sheet head data*/
    private Map<String, Object> getSheetHead(List<KeyValueDto> files, List<DynSearchFieldDto> searchKeys) {
        String searchKeysString = getSearchKeysString(searchKeys);
        String fileNames = "";
        for (KeyValueDto fileDto : files) {
            File file = new File(fileDto.getValue().toString());
            fileNames += file.getPath() + ";";
        }

        Map<String, Object> searchParams = new HashMap<>();
        searchParams.put("fileNames", fileNames);
        searchParams.put("searchKeysString", searchKeysString);

        return searchParams;
    }

    private Map<String, Object> getRuleDescription() {
        Map<String, Object> results = new HashMap<>();
        List<SpcConfigDto> spcConfigDtoList = new ArrayList<>();
        getAllWarningRules(null, spcConfigDtoList);
        results.put("ruleList", spcConfigDtoList);

        return results;
    }

    /*package the data of chart sheet which contains several charts in single item exporting*/
    private Map<String, Map> getChartPage(Map<String, Object> chartDatas, String chartName, List<String> chartsName, String imageDir, Map<String, ChartDto> guidesParams) {
        Map<String, Map> returnValue = new HashMap<>();
        Map<String, Map> tempReturnValue;
        Map<String, Object> searchParams = new HashMap<>();
        Map<String, Object> results = new HashMap<>();

        Map<String, Object> charts = new HashMap<>();

        if (chartsName.contains(AppConstant.CHART_TYPE_SPC_ND)) {
            tempReturnValue = getNDChart(chartDatas, imageDir, chartName);
            searchParams.putAll(tempReturnValue.get("searchParams"));
            results.putAll(tempReturnValue.get("results"));
            charts.putAll(tempReturnValue.get("exportCharts"));

        }
        //build control charts
        if (chartDatas.size() > 1) {
            tempReturnValue = (Map) getControlCharts(chartDatas, imageDir, chartsName, guidesParams, chartName);
            searchParams.putAll(tempReturnValue.get("searchParams"));
            results.putAll(tempReturnValue.get("results"));
            charts.putAll(tempReturnValue.get("exportCharts"));

        }
        if (chartsName.contains(AppConstant.CHART_TYPE_SPC_SCT)) {
            tempReturnValue = getSCTChart(chartDatas, imageDir, chartName);
            searchParams.putAll(tempReturnValue.get("searchParams"));
            results.putAll(tempReturnValue.get("results"));
            charts.putAll(tempReturnValue.get("exportCharts"));

        }

        searchParams.put("exportCharts", charts);

        returnValue.put("results", results);
        returnValue.put("searchParams", searchParams);

        return returnValue;
    }

    /*package the data of chart sheet which contains several charts for single item exporting*/
    private Map<String, Object> getTablePage(Map<String, Object> chartDatas, List<TestItemDto> testItemDtos) {
        Map<String, Object> results = new HashMap<>();
        ControlDataDto controlDataMRDto = (ControlDataDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_MR);
        ControlDataDto controlDataRUNDto = (ControlDataDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_RUN);

        double xbar = (controlDataRUNDto.getMeanData())[3];
        String[] xxbarValue = getXBarValue(testItemDtos, xbar);
        results.put("testItemDtos", testItemDtos);
        results.put("mrValue", controlDataMRDto.getY());
        results.put("xxbarValue", xxbarValue);

        return results;
    }

    /*for single item exporting*/
    private Map<String, Map> getNDChart(Map<String, Object> chartDatas, String imageDir, String chartName) {
        Map<String, Map> returnValue = new HashMap<>();
        Map<String, Object> searchParams = new HashMap<>();
        Map<String, Object> results = new HashMap<>();

        NormalDistributionDto normalDistributionDto = (NormalDistributionDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_ND);
        results.put(AppConstant.CHART_TYPE_SPC_ND, normalDistributionDto);

        ChartExporter chartExporter = new ChartExporterImpl(AppConstant.ANALYSIS_SPC_DIGNUM);
        String imagePath = imageDir + "/ndc" + new Date().getTime() + ".gif";
        chartExporter.exportSpcNDChart2Gif(normalDistributionDto, AppConstant.CHART_TYPE_SPC_ND, imagePath);

        Map<String, Object> charts = new HashMap<>();
        charts.put(AppConstant.CHART_TYPE_SPC_ND, imagePath);

        returnValue.put("exportCharts", charts);
        returnValue.put("results", results);
        returnValue.put("searchParams", searchParams);
        return returnValue;

    }

    /*for single item exporting*/
    private Map<String, Map> getSCTChart(Map<String, Object> chartDatas, String imageDir, String chartName) {
        Map<String, Map> returnValue = new HashMap<>();
        Map<String, Object> searchParams = new HashMap<>();
        Map<String, Object> results = new HashMap<>();

        ScatterDto scatterDto = (ScatterDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_SCT);
        if (scatterDto == null) {
            return null;
        }
        results.put(AppConstant.CHART_TYPE_SPC_SCT, scatterDto);

        ChartExporter chartExporter = new ChartExporterImpl(AppConstant.ANALYSIS_SPC_DIGNUM);
        String imagePath = imageDir + "/scatter" + new Date().getTime() + ".gif";
        chartExporter.exportSpcScatter2Gif(scatterDto, AppConstant.CHART_TYPE_SPC_SCT, imagePath);

        Map<String, Object> charts = new HashMap<>();
        charts.put(AppConstant.CHART_TYPE_SPC_SCT, imagePath);

        returnValue.put("exportCharts", charts);
        returnValue.put("results", results);
        returnValue.put("searchParams", searchParams);
        return returnValue;

    }

    /*for single item exporting*/
    private Map<String, Object> getControlCharts(Map<String, Object> chartDatas, String imageDir, List<String> chartsName, Map<String, ChartDto> guidesParams, String chartName) {
        Map<String, Object> returnValue = new HashMap<>();
        Map<String, Object> searchParams = new HashMap<>();
        Map<String, Object> results = new HashMap<>();
        //add control charts
        String[] controlChartsNameArray = {AppConstant.CHART_TYPE_SPC_RUN, AppConstant.CHART_TYPE_SPC_XBAR, AppConstant.CHART_TYPE_SPC_SD, AppConstant.CHART_TYPE_SPC_R, AppConstant.CHART_TYPE_SPC_MR};
        List<String> controlChartsNameList = new ArrayList<>();
        Collections.addAll(controlChartsNameList, controlChartsNameArray);
        Map<String, String> imagesPath = new HashMap<>();
        Map<String, ControlDataDto> controlDataDtos = new HashMap<>();
        String path;
//        Map<String, Object> charts = (Map) searchParams.get("exportCharts");
        Map<String, String> charts = new HashMap<>();
        returnValue.put("exportCharts", charts);
        for (String str : chartsName) {
            if (!controlChartsNameList.contains(str)) {
                continue;
            }
            ControlDataDto controlDataDto = (ControlDataDto) chartDatas.get(str);
            controlDataDto.setControlChartType(str);
            if (guidesParams != null) {
                ChartDto chartDto = guidesParams.get(str);
                if (str.equals(AppConstant.CHART_TYPE_SPC_RUN) && chartDto != null) {
                    List<Double> coupleGuidesParams;
                    coupleGuidesParams = guidesParams.get(str).getXline();
                    controlDataDto.setGuidesParams(coupleGuidesParams);
                } else if (str.equals(AppConstant.CHART_TYPE_SPC_MR) && chartDto != null) {
                    List<Double> coupleGuidesParams;
                    coupleGuidesParams = guidesParams.get(str).getXline();
                    controlDataDto.setGuidesParams(coupleGuidesParams);
                }
            }
            path = imageDir + "/" + str + new Date().getTime() + ".gif";
            charts.put(str, path);
            controlDataDtos.put(str, controlDataDto);
            imagesPath.put(str, path);
            results.put(str, chartDatas.get(str));
        }
        String chartTitle = chartName;
        ChartExporter chartExporter = new ChartExporterImpl(AppConstant.ANALYSIS_SPC_DIGNUM);
        ChartDto chartDto = new ChartDto();
        chartDto.setName(chartTitle);
        chartExporter.exportSpcAllRunChart2Gif(controlDataDtos, chartDto, imagesPath);

        returnValue.put("exportCharts", charts);
        returnValue.put("results", results);
        returnValue.put("searchParams", searchParams);
        return returnValue;

    }

    private boolean isBlank(String d) {
        if (StringUtils.isBlank(d)) {
            return true;
        }
        if (d.equalsIgnoreCase("N/A") || d.equalsIgnoreCase("-") || d.equalsIgnoreCase("nil") || d.equalsIgnoreCase("UnKown Line") || d.equalsIgnoreCase("_")) {
            return true;
        }

        return false;
    }

    /*extract data from multiple dtos as array*/
    private String[] getXBarValue(List<TestItemDto> testItemDtos, double xbar) {
        int length = testItemDtos.size();
        String[] xxbarValue = new String[length];
        for (int i = 0; i < length; i++) {
            if (testItemDtos.get(i) != null && testItemDtos.get(i).getNumeric() && !isBlank(testItemDtos.get(i).getValue())) {
                double value = (Double.valueOf(testItemDtos.get(i).getValue()) - xbar);
                xxbarValue[i] = String.format("%." + 6 + "f", value);
            } else {
                xxbarValue[i] = "-";
            }
        }
        return xxbarValue;
    }

    private String getSearchKeysString(List<DynSearchFieldDto> searchKeys) {
        String searchKeyString = "";
        if (searchKeys == null) {
            return null;
        }
        for (DynSearchFieldDto dto : searchKeys) {
            if (dto.isAdvance()) {
                searchKeyString = dto.getValue();
                break;
            } else {
                String token = dto.getToken();
                searchKeyString += dto.getName() + " " + token + " " + dto.getValue() + ";\t";
            }
        }
        return searchKeyString;
    }

    private Integer updateGroupNumber(String type) {
        SpcConfigDto spcConfigDto = systemService.findSpcConfigById(1L);
        Integer tempGroupNum = null;
        switch (type) {
            case "ndc":
                tempGroupNum = spcConfigDto.getSpcNdGroup();
                break;
            case "cc":
                tempGroupNum = spcConfigDto.getSpcCcGroup();
                break;
            default:
                break;
        }
        return tempGroupNum;
    }

    private String getTimeString() {
        DateTimeFormatter format = DateTimeFormat.forPattern("yyyyMMdd_HHmmss_SSS");
        DateTime now = new DateTime();

        return now.toString(format);
    }

    private List<DynSearchFieldDto> getSearchKeys(List<String> searchKeysObject, Set<String> searchKeyNames, SystemConfigDto systemConfigDto) {
        List<List<String>> searchKeysList;
        Set<String> searchKeys;
        if (searchKeysObject == null) {
            searchKeys = new LinkedHashSet<>(systemConfigDto.getSearchKeyKeys());
            searchKeyNames.addAll(searchKeys);
        } else {
            searchKeys = new LinkedHashSet<>();
            searchKeysList = jsonMapper.fromJson(searchKeysObject.toString(), List.class);
            searchKeysList.forEach(searchKeys::addAll);
            searchKeyNames.addAll(searchKeys);
        }
        List<String> allSearchKeys = Lists.newArrayList(searchKeys);

        List<DynSearchFieldDto> dynSearchFieldDtos = new ArrayList<>();
        allSearchKeys.forEach((searchKey) -> {
                    DynSearchFieldDto dynSearchFieldDto = new DynSearchFieldDto();
                    dynSearchFieldDto.setName(searchKey);
                    dynSearchFieldDtos.add(dynSearchFieldDto);
                }
        );
        return dynSearchFieldDtos;
    }

    /*get attribute keys from attributeKeysObject or systemConfigDto*/
    private List<String> getAttributeKeys(List<String> attributeKeysObject, SystemConfigDto systemConfigDto) {
        Set<String> attributeKeyNames = new HashSet<>();
        List<List<String>> attributeKeysList;
        Set<String> attributeKeys;
        if (attributeKeysObject == null) {
            attributeKeys = new LinkedHashSet<>(systemConfigDto.getAttributeKeys());
            attributeKeyNames.addAll(attributeKeys);
        } else {
            attributeKeys = new LinkedHashSet<>();
            attributeKeysList = jsonMapper.fromJson(attributeKeysObject.toString(), List.class);
            attributeKeysList.forEach(attributeKeys::addAll);
            attributeKeyNames.addAll(attributeKeys);
        }
        List<String> allAttributeKeys = new ArrayList<>(attributeKeys);
        return allAttributeKeys;
    }

    /*get test item from json*/
    private List<String> getTestItemKeys(List<String> itemNameObject) {
        List<List<String>> itemNameList = jsonMapper.fromJson(itemNameObject.toString(), List.class);
        Set<String> testItemNamesSet = new LinkedHashSet<>();
        itemNameList.forEach(testItemNamesSet::addAll);
        List<String> testItemNames = new ArrayList<>(testItemNamesSet);
        return testItemNames;

    }

    /*check all configKeys exist in the opposite file*/
    private String checkKeys(List<String> configKeys, List<String> fileKeys) {
        int fileKeysSize = fileKeys.size();
        for (int i = 0; i < fileKeysSize; i++) {
            if (configKeys.size() == 0) {
                break;
            }
            configKeys.remove(fileKeys.get(i));
        }
        String error = "";
        if (configKeys.size() > 0) {
            error = "";
            for (String keyName : configKeys) {
                error += keyName + "\n";
            }
        }

        return error;
    }

    /*get all test item names  from json*/
    private List<String> getAllColNames(List<List<String>> colNamesObject) {
        List<String> allColNames = new ArrayList<>();
        colNamesObject.forEach(allColNames::addAll);

        return allColNames;
    }

    /*export specified multiple items from a list of testItemName
    * new2
    * */
    private boolean exportMultipleNdcCharts(int startIndex, List<KeyValueDto> files, Map<String, SpcExportDto> spcExportDtos, List<String> testItemName, ExportManager exportManager,
                                            ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        Stream<SpcResultDto> spcStream = null;
        SpcExportBuilder spcExportBuilder = new SpcExportBuilder();
        SpcExportWorker spcExportWorker = new SpcExportWorker();
        spcExportWorker.setCurWrittenItemNum(startIndex);
        int rate = exportManager.getRate(), readPieceSize = exportManager.getReadPieceSize();
        String dirName = exportManager.getDirName(),
                dirSavePath = exportManager.getDirSavePath(),
                imageDir = exportManager.getImageDir() + "/" + Thread.currentThread().getId();

        int excelCapacity = readPieceSize * rate;
        int length = testItemName.size();
        int exportTimes = 0;
        final int deleteImageLimitTimes = 10;
        List<String> dirsToDelete = Lists.newArrayList();
//        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        try {
            double progress = (75 / (Math.ceil((double) length / readPieceSize)) == 75.0 ? 40.0 : (75 / (Math.ceil((double) length / readPieceSize))));
            int process = 25;
            int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());
            //This circle load and export data piece by piece
            for (int i = 0; i < length; i += readPieceSize) {
                // exportThread is used to check if current thread has been interrupted
                Thread exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                exportTimes++;
                Map<String, Object> searchParams = new HashMap<>();
                Map<String, Map<String, String>> charts = new LinkedHashMap<>();
                List<ExportBaseObject> exportList = new ArrayList<>();
                Map<String, Object> resultData = new HashMap<>();
                Map<String, Object> testItemMap = new HashMap<>();
                List<String> imagesPath = new ArrayList<>();

                List<String> partTestItemName;
                try {
                    partTestItemName = testItemName.subList(i, i + readPieceSize);
                } catch (IndexOutOfBoundsException e) {
                    partTestItemName = testItemName.subList(i, testItemName.size());
                }
                try {
                    spcStream = getStatisticResult(files, spcExportDtos, partTestItemName).stream();
                } catch (NullPointerException e) {
                    logger.debug("Result is empty");
                    return false;
                }
                int temp = i;
                int[] counter = {0};
                String currentImageDir = imageDir + "_" + getTimeString();
                if (spcStream != null) {
                    //final int finalProcess = process;
                    spcStream.forEach(item -> {
                        Thread tempExportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                        if (tempExportThread == null) {
                            return;
                        }
                        String itemName = item.getTestItemName();
                        SpcExportDto spcExportDto = spcExportDtos.get(itemName);
                        //processHelper.postProcess(finalProcess, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA) + ":" + itemName);
                        SpcResultDto spcResultDto = item;
                        if (counter[0] == 0) {
                            searchParams.clear();
                            charts.clear();
                            exportList.clear();
                            resultData.clear();
                            testItemMap.clear();
                            searchParams.put("primaryKey", spcResultDto.getPrimaryKey());
                        }
                        List<TestItemDto> testItemDtos = spcResultDto.getTestItems();
                        if (spcExportDto.getExcludeItems() != null) {
                            List<TestItemDto> toRemoveItems = spcExportDto.getExcludeItems()
                                    .stream()
                                    .map(tIDto -> testItemDtos.get(tIDto.getId().intValue())).collect(Collectors.toList());
                            testItemDtos.removeAll(toRemoveItems);
                        }
                        if (testItemDtos.isEmpty()) {
                            return;
                        }
                        Map<String, Object> chartDatas = spcResultDto.getChartData();
                        NormalDistributionDto normalDistributionDto = (NormalDistributionDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_ND);

                        Double lsl, usl;
                        try {
                            lsl = Double.parseDouble(testItemDtos.get(0).getLsl());
                        } catch (NullPointerException e) {
                            lsl = null;
                        }
                        try {
                            usl = Double.parseDouble(testItemDtos.get(0).getUsl());
                        } catch (NullPointerException e) {
                            usl = null;
                        }
                        Map<String, String> itemCharts = new HashMap<>();
                        FileUtils.createDir(currentImageDir);
                        List<String> chartNames = spcExportDto.getChartNames();
                        //draw NDC chart
                        if (chartNames.contains(AppConstant.CHART_TYPE_SPC_ND)) {

                            Map<String, String> ndcChartPath = getNDChart(chartDatas, currentImageDir, itemName).get("exportCharts");
                            if (ndcChartPath != null) {
                                itemCharts.putAll(ndcChartPath);
                            }
                        }
                        //draw scatter chart
                        if (chartNames.contains(AppConstant.CHART_TYPE_SPC_SCT)) {
                            Map<String, String> scatterChartPath = getValueFromMap(getSCTChart(chartDatas, currentImageDir, itemName), "exportCharts");
                            if (scatterChartPath != null) {
                                itemCharts.putAll(scatterChartPath);
                            }
                        }
                        //draw control charts
                        ChartDto chartDto = new ChartDto();
                        chartDto.setName(itemName);
                        Map<String, ChartDto> guidesParams = spcExportDto.getGuidesParams();
                        Map<String, String> controlChartsPath = (Map<String, String>) getControlCharts(chartDatas, currentImageDir, chartNames, guidesParams, itemName).get("exportCharts");
                        if (controlChartsPath != null) {
                            itemCharts.putAll(controlChartsPath);
                        }
                        charts.put(itemName, itemCharts);

                        double xbar = normalDistributionDto.getStatisticsDto().getAvgValue();
                        String[] xxbarValue = getXBarValue(testItemDtos, xbar);

                        List<Double> dataList = new ArrayList<>(4000);
                        for (TestItemDto testItemDto : testItemDtos) {
                            String value = testItemDto.getValue();
                            if (testItemDto.getNumeric() && !isBlank(value)) {
                                dataList.add(Double.parseDouble(value));
                            }
                        }
                        int dataLength = dataList.size();
                        double[] data = new double[dataLength];
                        for (int k = 0; k < dataLength; k++) {
                            data[k] = dataList.get(k);
                        }
                        ExportBaseObject exportObj = new ExportBaseObject();
                        exportObj.setAnalyseData(data);
                        exportObj.setFilePath(null);
                        exportObj.setItemName(itemName);
                        exportObj.setUsl(usl);
                        exportObj.setLsl(lsl);
                        double[] mrValue = (double[]) normalDistributionDto.getOtherAttribute().get("claData");
                        exportObj.setMrValue(mrValue);
                        exportObj.setXxbarValue(xxbarValue);
                        exportList.add(exportObj);

                        resultData.put(itemName, chartDatas);
                        testItemMap.put(itemName, testItemDtos);
                        counter[0]++;
                    });
                }
                if (spcStream != null) {
                    spcStream.close();
                    spcStream = null;
                }

                logger.debug("Loading export data.");
                searchParams.put("charts", exportList);
                searchParams.put("exportCharts", charts);
                resultData.put("ruleList", getRuleDescription().get("ruleList"));
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));


//                searchParams.putAll(getSheetHead(files, searchKeys));

                spcExportWorker.setExcelItemCapacity(excelCapacity);
                String excelPath = dirSavePath + "/" + dirName + "_" + ((i + startIndex) / excelCapacity) + ".xlsx";
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL) + ":" + excelPath);
                spcExportWorker.initWorkbook(excelPath);
                spcExportWorker.setDigNum(digitNum);
                spcExportWorker.buildSPCMultiItem(searchParams, resultData, testItemMap);
                spcExportWorker.setCurWrittenItemNum(i + startIndex);
                searchParams.clear();
                resultData.clear();
                testItemMap.clear();
                logger.debug("wait for images ready");
                boolean imagesNotReady = true;
                while (imagesNotReady) {
                    imagesPath.forEach(path -> {
                        if (new File(path).exists()) {
                            imagesPath.remove(path);
                        }
                    });
                    if (imagesPath.size() == 0) {
                        imagesNotReady = false;
                    }
                }
                logger.debug("Loading export data done.");
                // exportThread is used to check if current thread has been interrupted
                exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                spcExportBuilder.drawSpcExcel(excelPath, spcExportWorker);
                spcExportBuilder.clear();
                spcExportWorker.cleanExportWorker();
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL_DONE));
                if (exportTimes % deleteImageLimitTimes == 0) {
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
                    dirsToDelete.forEach(dir -> FileUtils.deleteDir(dir));
                    dirsToDelete.clear();
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
                }
                dirsToDelete.add(currentImageDir);
                process += progress;
//                spcResultDtos = preReadTask.get();
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return true;
    }

    /*export specified multiple items from a list of testItemName
    * new
    * */
    private boolean exportMultipleNdcChart(int startIndex, List<KeyValueDto> files, Map<String, List<DynSearchFieldDto>> searchKeys, List<String> testItemName, ExportManager exportManager,
                                           ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        Stream<SpcResultDto> spcStream = null;
        SpcExportBuilder spcExportBuilder = new SpcExportBuilder();
        SpcExportWorker spcExportWorker = new SpcExportWorker();
        spcExportWorker.setCurWrittenItemNum(startIndex);
        int rate = exportManager.getRate(), readPieceSize = exportManager.getReadPieceSize();
        String dirName = exportManager.getDirName(),
                dirSavePath = exportManager.getDirSavePath(),
                imageDir = exportManager.getImageDir() + "/" + Thread.currentThread().getId();

        int excelCapacity = readPieceSize * rate;
        int length = testItemName.size();
        int exportTimes = 0;
        final int deleteImageLimitTimes = 10;
        List<String> dirsToDelete = Lists.newArrayList();
//        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        try {
            double progress = (75 / (Math.ceil((double) length / readPieceSize)) == 75.0 ? 40.0 : (75 / (Math.ceil((double) length / readPieceSize))));
            int process = 25;
            int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());
            //This circle load and export data piece by piece
            for (int i = 0; i < length; i += readPieceSize) {
                // exportThread is used to check if current thread has been interrupted
                Thread exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                exportTimes++;
                Map<String, Object> searchParams = new HashMap<>();
                Map<String, Map<String, String>> charts = new LinkedHashMap<>();
                List<ExportBaseObject> exportList = new ArrayList<>(4000);
                Map<String, Object> resultData = new HashMap<>();
                Map<String, Object> testItemMap = new HashMap<>();
                List<String> imagesPath = new ArrayList<>();

                List<String> partTestItemName;
                try {
                    partTestItemName = testItemName.subList(i, i + readPieceSize);
                } catch (IndexOutOfBoundsException e) {
                    partTestItemName = testItemName.subList(i, testItemName.size());
                }
                try {
                    spcStream = getStatisticResult(files, searchKeys, partTestItemName, exportManager.getExcludeItems(), exportManager.getParam(), exportManager.getChartNames()).stream();
                } catch (NullPointerException e) {
                    logger.debug("Result is empty");
                    return false;
                }
                int temp = i;
                int[] counter = {0};
                String currentImageDir = imageDir + "_" + getTimeString();
                if (spcStream != null) {
                    //final int finalProcess = process;
                    spcStream.forEach(item -> {
                        Thread tempExportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                        if (tempExportThread == null) {
                            return;
                        }
                        String itemName = testItemName.get(temp + counter[0]);
                        //processHelper.postProcess(finalProcess, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA) + ":" + itemName);
                        SpcResultDto spcResultDto = item;
                        if (counter[0] == 0) {
                            searchParams.clear();
                            charts.clear();
                            exportList.clear();
                            resultData.clear();
                            testItemMap.clear();
                            searchParams.put("primaryKey", spcResultDto.getPrimaryKey());
                        }
                        List<TestItemDto> testItemDtos = spcResultDto.getTestItems();
                        List<TestItemDto> excludeItems = exportManager.getExcludeItems().get(itemName);
                        if (testItemDtos.isEmpty()) {
                            return;
                        }
                        for (TestItemDto tIDto : excludeItems) {
                            testItemDtos.remove(tIDto.getId().intValue());
                        }
                        Map<String, Object> chartDatas = spcResultDto.getChartData();
                        NormalDistributionDto normalDistributionDto = (NormalDistributionDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_ND);

                        Double lsl, usl;
                        try {
                            lsl = Double.parseDouble(testItemDtos.get(0).getLsl());
                        } catch (NullPointerException e) {
                            lsl = null;
                        }
                        try {
                            usl = Double.parseDouble(testItemDtos.get(0).getUsl());
                        } catch (NullPointerException e) {
                            usl = null;
                        }
                        Map<String, String> itemCharts = new HashMap<>();
                        FileUtils.createDir(currentImageDir);
                        List<String> chartNames = getValueFromMap(exportManager.getChartNames(), itemName);
                        //draw NDC chart
                        if (chartNames.contains(AppConstant.CHART_TYPE_SPC_ND)) {

                            Map<String, String> ndcChartPath = getNDChart(chartDatas, currentImageDir, itemName).get("exportCharts");
                            if (ndcChartPath != null) {
                                itemCharts.putAll(ndcChartPath);
                            }
                        }
                        //draw scatter chart
                        if (chartNames.contains(AppConstant.CHART_TYPE_SPC_SCT)) {
                            Map<String, String> scatterChartPath = getValueFromMap(getSCTChart(chartDatas, currentImageDir, itemName), "exportCharts");
                            if (scatterChartPath != null) {
                                itemCharts.putAll(scatterChartPath);
                            }
                        }
                        //draw control charts
                        ChartDto chartDto = new ChartDto();
                        chartDto.setName(itemName);
                        Map<String, ChartDto> guidesParams = getValueFromMap(exportManager.getGuidesParams(), itemName);
                        Map<String, String> controlChartsPath = (Map<String, String>) getControlCharts(chartDatas, currentImageDir, chartNames, guidesParams, itemName).get("exportCharts");
                        if (controlChartsPath != null) {
                            itemCharts.putAll(controlChartsPath);
                        }
                        charts.put(itemName, itemCharts);

                        double xbar = normalDistributionDto.getStatisticsDto().getAvgValue();
                        String[] xxbarValue = getXBarValue(testItemDtos, xbar);

                        List<Double> dataList = new ArrayList<>(4000);
                        for (TestItemDto testItemDto : testItemDtos) {
                            String value = testItemDto.getValue();
                            if (testItemDto.getNumeric() && !isBlank(value)) {
                                dataList.add(Double.parseDouble(value));
                            }
                        }
                        int dataLength = dataList.size();
                        double[] data = new double[dataLength];
                        for (int k = 0; k < dataLength; k++) {
                            data[k] = dataList.get(k);
                        }
                        ExportBaseObject exportObj = new ExportBaseObject();
                        exportObj.setAnalyseData(data);
                        exportObj.setFilePath(null);
                        exportObj.setItemName(itemName);
                        exportObj.setUsl(usl);
                        exportObj.setLsl(lsl);
                        double[] mrValue = (double[]) normalDistributionDto.getOtherAttribute().get("claData");
                        exportObj.setMrValue(mrValue);
                        exportObj.setXxbarValue(xxbarValue);
                        exportList.add(exportObj);

                        resultData.put(itemName, chartDatas);
                        testItemMap.put(itemName, testItemDtos);
                        counter[0]++;
                    });
                }
                if (spcStream != null) {
                    spcStream.close();
                    spcStream = null;
                }

                logger.debug("Loading export data.");
                searchParams.put("charts", exportList);
                searchParams.put("exportCharts", charts);
                searchParams.put("excludedItems", exportManager.getExcludeItems());
                resultData.put("ruleList", getRuleDescription().get("ruleList"));
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));


//                searchParams.putAll(getSheetHead(files, searchKeys));

                spcExportWorker.setExcelItemCapacity(excelCapacity);
                String excelPath = dirSavePath + "/" + dirName + "_" + ((i + startIndex) / excelCapacity) + ".xlsx";
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL) + ":" + excelPath);
                spcExportWorker.initWorkbook(excelPath);
                spcExportWorker.setDigNum(digitNum);
                spcExportWorker.buildSPCMultiItem(searchParams, resultData, testItemMap);
                spcExportWorker.setCurWrittenItemNum(i + startIndex);
                searchParams.clear();
                resultData.clear();
                testItemMap.clear();
                logger.debug("wait for images ready");
                boolean imagesNotReady = true;
                while (imagesNotReady) {
                    imagesPath.forEach(path -> {
                        if (new File(path).exists()) {
                            imagesPath.remove(path);
                        }
                    });
                    if (imagesPath.size() == 0) {
                        imagesNotReady = false;
                    }
                }
                logger.debug("Loading export data done.");
                // exportThread is used to check if current thread has been interrupted
                exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                spcExportBuilder.drawSpcExcel(excelPath, spcExportWorker);
                spcExportBuilder.clear();
                spcExportWorker.cleanExportWorker();
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL_DONE));
                if (exportTimes % deleteImageLimitTimes == 0) {
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
                    dirsToDelete.forEach(dir -> FileUtils.deleteDir(dir));
                    dirsToDelete.clear();
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
                }
                dirsToDelete.add(currentImageDir);
                process += progress;
//                spcResultDtos = preReadTask.get();
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return true;
    }

    /*export specified multiple items from a list of testItemName*/
    private boolean exportMultipleNdcChart(int startIndex, List<KeyValueDto> files, List<DynSearchFieldDto> searchKeys, List<String> testItemName, ExportManager exportManager,
                                           ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        ChartExporter chartExporter = new ChartExporterImpl(AppConstant.ANALYSIS_SPC_DIGNUM);
        Stream<SpcResultDto> spcStream = null;
        SpcExportBuilder spcExportBuilder = new SpcExportBuilder();
        int rate = exportManager.getRate(), readPieceSize = exportManager.getReadPieceSize();
        String dirName = exportManager.getDirName(),
                dirSavePath = exportManager.getDirSavePath(),
                imageDir = exportManager.getImageDir() + "/" + Thread.currentThread().getId();

        int excelCapacity = readPieceSize * rate;
        int length = testItemName.size();
        int exportTimes = 0;
        final int deleteImageLimitTimes = 10;
        List<String> dirsToDelete = Lists.newArrayList();
//        List<SpcResultDto> spcResultDtos = new ArrayList<>();
        try {
            double progress = (75 / (Math.ceil((double) length / readPieceSize)) == 75.0 ? 40.0 : (75 / (Math.ceil((double) length / readPieceSize))));
            int process = 25;
            int digitNum = Integer.parseInt(systemService.findSpcConfigById(digitId).getValue());
            SpcExportWorker spcExportWorker = new SpcExportWorker();
            spcExportWorker.setCurWrittenItemNum(startIndex);
            //This circle load and export data piece by piece
            for (int i = 0; i < length; i += readPieceSize) {
                // exportThread is used to check if current thread has been interrupted
                Thread exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                exportTimes++;
                Map<String, Object> searchParams = new HashMap<>();
                List<KeyValueDto> charts = new ArrayList<>();
                List<ExportBaseObject> exportList = new ArrayList<>(4000);
                Map<String, Object> resultData = new HashMap<>();
                Map<String, Object> testItemMap = new HashMap<>();
                List<String> imagesPath = new ArrayList<>();

                List<String> partTestItemName;
                try {
                    partTestItemName = testItemName.subList(i, i + readPieceSize);
                } catch (IndexOutOfBoundsException e) {
                    partTestItemName = testItemName.subList(i, testItemName.size());
                }
                try {
                    spcStream = getStatisticResult(files, searchKeys, partTestItemName, null, null, null).stream();
                } catch (NullPointerException e) {
                    logger.debug("Result is empty");
                    return false;
                }
                int temp = i;
                int[] counter = {0};
                String currentImageDir = imageDir + "_" + getTimeString();
                if (spcStream != null) {
                    //final int finalProcess = process;
                    spcStream.forEach(item -> {
                        Thread tempExportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                        if (tempExportThread == null) {
                            return;
                        }
                        String itemName = testItemName.get(temp + counter[0]);
                        //processHelper.postProcess(finalProcess, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA) + ":" + itemName);
                        SpcResultDto spcResultDto = item;
                        if (counter[0] == 0) {
                            searchParams.clear();
                            charts.clear();
                            exportList.clear();
                            resultData.clear();
                            testItemMap.clear();
                            searchParams.put("primaryKey", spcResultDto.getPrimaryKey());
                        }
                        List<TestItemDto> testItemDtos = spcResultDto.getTestItems();
                        if (testItemDtos.isEmpty()) {
                            return;
                        }
                        Map<String, Object> chartDatas = spcResultDto.getChartData();
                        NormalDistributionDto normalDistributionDto = (NormalDistributionDto) chartDatas.get(AppConstant.CHART_TYPE_SPC_ND);
                        Double lsl, usl;
                        try {
                            lsl = Double.parseDouble(testItemDtos.get(0).getLsl());
                        } catch (NullPointerException e) {
                            lsl = null;
                        }
                        try {
                            usl = Double.parseDouble(testItemDtos.get(0).getUsl());
                        } catch (NullPointerException e) {
                            usl = null;
                        }
                        KeyValueDto itemInfo = new KeyValueDto();
                        itemInfo.setKey(itemName);
                        String imagePath = currentImageDir + "/" + counter[0] + ".gif";
                        FileUtils.createDir(currentImageDir);
                        itemInfo.setValue(imagePath);
                        chartExporter.exportSpcNDChart2Gif(normalDistributionDto, itemName, imagePath);
                        charts.add(itemInfo);

                        double xbar = normalDistributionDto.getStatisticsDto().getAvgValue();
                        String[] xxbarValue = getXBarValue(testItemDtos, xbar);

                        List<Double> dataList = new ArrayList<>(4000);
                        for (TestItemDto testItemDto : testItemDtos) {
                            String value = testItemDto.getValue();
                            if (testItemDto.getNumeric() && !isBlank(value)) {
                                dataList.add(Double.parseDouble(value));
                            }
                        }
                        int dataLength = dataList.size();
                        double[] data = new double[dataLength];
                        for (int k = 0; k < dataLength; k++) {
                            data[k] = dataList.get(k);
                        }
                        ExportBaseObject exportObj = new ExportBaseObject();
                        exportObj.setAnalyseData(data);
                        exportObj.setFilePath(imagePath);
                        exportObj.setItemName(itemName);
                        exportObj.setUsl(usl);
                        exportObj.setLsl(lsl);
                        double[] mrValue = (double[]) normalDistributionDto.getOtherAttribute().get("claData");
                        exportObj.setMrValue(mrValue);
                        exportObj.setXxbarValue(xxbarValue);
                        exportList.add(exportObj);

                        resultData.put(itemName, normalDistributionDto);
                        testItemMap.put(itemName, testItemDtos);
                        counter[0]++;
                    });
                }
                if (spcStream != null) {
                    spcStream.close();
                    spcStream = null;
                }


                logger.debug("Loading export data.");
                searchParams.put("charts", exportList);
                searchParams.put("exportCharts", charts);
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));


                searchParams.putAll(getSheetHead(files, searchKeys));

                spcExportWorker.setExcelItemCapacity(excelCapacity);
                String excelPath = dirSavePath + "/" + dirName + "_" + ((i + startIndex) / excelCapacity) + ".xlsx";
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL) + ":" + excelPath);
                spcExportWorker.initWorkbook(excelPath);
                spcExportWorker.setDigNum(digitNum);
                spcExportWorker.buildSPCMultiItems(searchParams, resultData, testItemMap);
                searchParams.clear();
                resultData.clear();
                testItemMap.clear();
                boolean imagesNotReady = true;
                while (imagesNotReady) {
                    imagesPath.forEach(path -> {
                        if (new File(path).exists()) {
                            imagesPath.remove(path);
                        }
                    });
                    if (imagesPath.size() == 0) {
                        imagesNotReady = false;
                    }
                }
                logger.debug("Loading export data done.");
                // exportThread is used to check if current thread has been interrupted
                exportThread = exportThreadManager.getSpcExportThreads().get(Thread.currentThread().getName());
                if (exportThread == null) {
                    return false;
                }
                exportThread = null;
                spcExportBuilder.drawSpcExcel(excelPath, spcExportWorker);
                spcExportWorker.cleanExportWorker();
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_EXCEL_DONE));
                if (exportTimes % deleteImageLimitTimes == 0) {
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
                    dirsToDelete.forEach(dir -> FileUtils.deleteDir(dir));
                    dirsToDelete.clear();
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));
                }
                dirsToDelete.add(currentImageDir);
                process += progress;
//                spcResultDtos = preReadTask.get();
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return true;
    }

    private <T> T getValueFromMap(Map<String, T> map, String key) {
        if (map == null) {
            return null;
        } else {
            return map.get(key);
        }
    }


}

/**
 * to reduce input params in method "exportMultipleNdcChart",package necessary data
 */
class ExportManager {
    private String dirName;
    private String dirSavePath;
    private String imageDir;
    private int rate, readPieceSize;
    private Map<String, List<String>> chartNames;
    private Map<String, Map<String, ChartDto>> guidesParams;
    private Map<String, List<TestItemDto>> excludeItems;
    private Map<String, NDStandardDto> param;

    ExportManager(String dirName, String dirSavePath, String imageDir, int rate, int readPieceSize) {
        this.dirName = dirName;
        this.dirSavePath = dirSavePath;
        this.imageDir = imageDir;
        this.rate = rate;
        this.readPieceSize = readPieceSize;
    }

    ExportManager(String dirName, String dirSavePath, String imageDir, int rate, int readPieceSize, Map<String, List<String>> chartNames,
                  Map<String, Map<String, ChartDto>> guidesParams, Map<String, List<TestItemDto>> excludeItems, Map<String, NDStandardDto> param) {
        this.dirName = dirName;
        this.dirSavePath = dirSavePath;
        this.imageDir = imageDir;
        this.rate = rate;
        this.readPieceSize = readPieceSize;
        this.chartNames = chartNames;
        this.guidesParams = guidesParams;
        this.excludeItems = excludeItems;
        this.param = param;
    }

    public String getDirName() {
        return dirName;
    }

    public void setDirName(String dirName) {
        this.dirName = dirName;
    }

    public String getDirSavePath() {
        return dirSavePath;
    }

    public void setDirSavePath(String dirSavePath) {
        this.dirSavePath = dirSavePath;
    }

    public String getImageDir() {
        return imageDir;
    }

    public void setImageDir(String imageDir) {
        this.imageDir = imageDir;
    }

    public int getRate() {
        return rate;
    }

    public void setRate(int rate) {
        this.rate = rate;
    }

    public int getReadPieceSize() {
        return readPieceSize;
    }

    public void setReadPieceSize(int readPieceSize) {
        this.readPieceSize = readPieceSize;
    }

    public Map<String, List<String>> getChartNames() {
        return chartNames;
    }

    public void setChartNames(Map<String, List<String>> chartNames) {
        this.chartNames = chartNames;
    }

    public Map<String, Map<String, ChartDto>> getGuidesParams() {
        return guidesParams;
    }

    public void setGuidesParams(Map<String, Map<String, ChartDto>> guidesParams) {
        this.guidesParams = guidesParams;
    }

    public Map<String, List<TestItemDto>> getExcludeItems() {
        return excludeItems;
    }

    public void setExcludeItems(Map<String, List<TestItemDto>> excludeItems) {
        this.excludeItems = excludeItems;
    }

    public Map<String, NDStandardDto> getParam() {
        return param;
    }

    public void setParam(Map<String, NDStandardDto> param) {
        this.param = param;
    }

}


